#include "BayesianFilterSerializer.h"
#include "BayesianHelper.h"
#include "Configuration.h"
#include "Feed.h"
#include "Log.h"
#include "MainThread.h"

#include "unistd.h"
#include "sys/types.h"
#include "signal.h"


bool MainThread::dirty = false;
bool MainThread::cleanRequested = false;
bool MainThread::politeStop = false;


string EnumTags(set<string> tags)
{
	string retval;

	set<string>::iterator iter;
	for ( iter = tags.begin() ; iter != tags.end() ; iter++ )
	{
		retval += *iter;
		retval += "; ";
	}	
	
	return retval;
}

void MainThread::ClasificarNoticia(News& news,TCId feedID,TCId newsID)
{
	LOG2("    ",news.getTitle());
	set<string> words = BayesianHelper::ParseWords(news.getTitle() +
		" " + news.getDescription());

	set<BayesianFilterResult*> result = state->bayesianFilter->Classify(words);
	
	set<string> positiveTags;
	set<string> negativeTags;
	set<string> missingTags;
	set<BayesianFilterResult*>::iterator iterResult;
	for ( iterResult = result.begin() ; iterResult != result.end() ; iterResult++ )
	{
		if      ( (*iterResult)->HitRatio < BAYES_HITRATIO_THRESHOLD )
			missingTags.insert((*iterResult)->Tag);
		else if ( (*iterResult)->Witness  > BAYES_WITNESS_THRESHOLD )
			positiveTags.insert((*iterResult)->Tag);
		else if ( (*iterResult)->Witness  < BAYES_WITNESS_THRESHOLD )
			negativeTags.insert((*iterResult)->Tag);
		else
			missingTags.insert((*iterResult)->Tag);
			
		delete (*iterResult);
	}

	if ( !positiveTags.empty() || !negativeTags.empty() )
	{
		state->bayesianFilter->Retrain(words,negativeTags,positiveTags);
		MainThread::SetDirty();
	}
	
	LOG2("      Positive tags: ",EnumTags(positiveTags));
	LOG2("      Negative tags: ",EnumTags(negativeTags));
	LOG2("      Unclassified tags: ",EnumTags(missingTags));
	
	list<TCId> positiveList;
	list<TCId> negativeList;
	set<string>::iterator iterTags;
	for ( iterTags = negativeTags.begin() ; iterTags != negativeTags.end() ; iterTags++ )
	{
		string tagName = *iterTags;
		negativeList.push_back(state->mapaCategorias->BuscarCategoria(tagName));
	}
	for ( iterTags = positiveTags.begin() ; iterTags != positiveTags.end() ; iterTags++ )
	{
		string tagName = *iterTags;
		TCId tagID = (TCId) state->mapaCategorias->BuscarCategoria(tagName);
		positiveList.push_back(tagID);
		state->tagsManager->AgregarNoticia(tagID,feedID,newsID,news.getDate());
	}
	
	state->newsManager->cambiarListas(feedID,newsID,positiveList,negativeList);
}


void MainThread::AgregarNoticiasFeed(Feed& feed)
{
	if (feed.begin() == feed.end())
	{
		LOG1("    Feed is up to date.");
	}
	else
	{
		LOG1("    News found...");
		LOG1("    Adding news to manager...");
		state->giantLock->Enter();
	
		Feed::iterator iterNews;
		for ( iterNews = feed.begin() ; iterNews != feed.end() ; iterNews++ )
		{
			TCId newsID = state->newsManager->AgregarNews(feed.getId(),*iterNews);
			
			if ( newsID == -1 )
				LOG3("    Error adding news '",iterNews->getTitle(),"' to manager.");	
			else
				ClasificarNoticia(*iterNews,feed.getId(),newsID);
		}
	
		state->giantLock->Release();
	}
}


void MainThread::ParseFeeds(Parser& parser)
{
	state->giantLock->Enter();
        list<FeedItemConsulta> feeds = state->feedManager->ConsultarFeed();
	state->giantLock->Release();
		
	list<FeedItemConsulta>::iterator iter;
        for ( iter = feeds.begin() ; iter != feeds.end() ; iter++ )
	{
		LOG4("  Reading (ID=",(*iter).getId(),") ",(*iter).getURL());
			
		Feed feed((*iter).getId(),(*iter).getURL(),(*iter).getPosicionGlobal());
		unsigned int parseResult = parser.Parse((*iter).getURL(),feed);
	
		if      ( parseResult == Parser::feedupdated )
			LOG1("    Feed is up to date.");
		else if ( parseResult == Parser::feedbad )
			LOG1("    Cannot read feed: bad syntax");
		else if ( parseResult == Parser::nofeed )
			LOG1("    Cannot read feed: no feed");
		else if ( parseResult == Parser::nocommand )
			LOG1("    Cannot read feed: no command");
		else if ( parseResult == Parser::feedok )
		{
			AgregarNoticiasFeed(feed);
			state->feedManager->ActualizarFechaFeed((*iter).getPosicionGlobal());
		}
		else 
			LOG2("   Cannot read feed: unknown response ",parseResult);
	}					

}


void MainThread::ReclasificarNoticia(NewsItemConsulta& news)
{
	LOG2("    ",news.getTitulo());

	bool listsChanged = false;
	set<TCId> classifiedTags;
	list<TCId>* lstPositive = news.getListaPositivosReferencia();
	list<TCId>* lstNegative = news.getListaNegativosReferencia();
	list<TCId>::iterator iterClass;
	for ( iterClass = lstPositive->begin() ; iterClass != lstPositive->end() ; iterClass++ )
		classifiedTags.insert(*iterClass);
	for ( iterClass = lstNegative->begin() ; iterClass != lstNegative->end() ; iterClass++ )
		classifiedTags.insert(*iterClass);
	
	set<string> words = BayesianHelper::ParseWords(news.getTitulo() +
			                " " + news.getDescripcion());

	state->giantLock->Enter();
        set<BayesianFilterResult*> result = state->bayesianFilter->Classify(words);

        set<string> positiveTags;
        set<string> negativeTags;
        set<string> missingTags;
        set<BayesianFilterResult*>::iterator iterResult;
        for ( iterResult = result.begin() ; iterResult != result.end() ; iterResult++ )
        {
		unsigned int tagId = state->mapaCategorias->BuscarCategoria((*iterResult)->Tag);
		if ( tagId != 0 && classifiedTags.find(tagId) == classifiedTags.end() )
		{
		       	if      ( (*iterResult)->HitRatio < BAYES_HITRATIO_THRESHOLD )
				missingTags.insert((*iterResult)->Tag);
			else if ( (*iterResult)->Witness  > BAYES_WITNESS_THRESHOLD )
			        positiveTags.insert((*iterResult)->Tag);
			else if ( (*iterResult)->Witness  < BAYES_WITNESS_THRESHOLD )
		        	negativeTags.insert((*iterResult)->Tag);
			else
				missingTags.insert((*iterResult)->Tag);
		}
    
		delete (*iterResult);
	}

	if ( !positiveTags.empty() || !negativeTags.empty() )
        {
                state->bayesianFilter->Retrain(words,negativeTags,positiveTags);
		MainThread::SetDirty();
		listsChanged = true;

		LOG2("      Positive tags: ",EnumTags(positiveTags));
		LOG2("      Negative tags: ",EnumTags(negativeTags));
		LOG2("      Unclassified tags: ",EnumTags(missingTags));
	}
	else
		LOG1("      No new positive or negative tags to add.");

	set<string>::iterator iterTags;
        for ( iterTags = negativeTags.begin() ; iterTags != negativeTags.end() ; iterTags++ )
        {
                string tagName = *iterTags;
                lstNegative->push_back(state->mapaCategorias->BuscarCategoria(tagName));
        }

        for ( iterTags = positiveTags.begin() ; iterTags != positiveTags.end() ; iterTags++ )
	{
	        string tagName = *iterTags;
	        TCId tagID = (TCId) state->mapaCategorias->BuscarCategoria(tagName);
	        lstPositive->push_back(tagID);
	        state->tagsManager->AgregarNoticia(tagID,news.getFeedId(),
				news.getPosicion(),news.getFecha());
	}	

	if (listsChanged)
	{
        	state->newsManager->cambiarListas
			(news.getFeedId(),news.getPosicion(),*lstPositive,*lstNegative);
	}

	state->giantLock->Release();
}


void MainThread::Reclasificar ()
{
	LOG1("Starting reclassification...");

	state->giantLock->Enter();
	list<FeedItemConsulta> feeds = state->feedManager->ConsultarFeed();
	state->giantLock->Release();

	list<FeedItemConsulta>::iterator iter;
	for ( iter = feeds.begin() ; iter != feeds.end() ; iter++ )
	{
		map<TCId,unsigned int>::iterator hasKey = reclasPos.find(iter->getId());
		if ( hasKey == reclasPos.end() )
			reclasPos.insert(make_pair(iter->getId(),0));

		state->giantLock->Enter();
		int count = state->newsManager->getCantidadNoticias(iter->getId());
		state->giantLock->Release();

		TCPosicionEnLista pos = reclasPos[iter->getId()];
		if ( count < pos ) 
		{
			pos = 0;
			reclasPos[iter->getId()] = 0;
		}
		LOG5("  ",iter->getURL()," (",pos,")");
		
		state->giantLock->Enter();
		list<NewsItemConsulta> news = state->newsManager->consultarNews
				(pos,RECLASSIFY_STEP,iter->getId());
		state->giantLock->Release();

		list<NewsItemConsulta>::iterator iterNews;
		for ( iterNews = news.begin() ; iterNews != news.end() ; iterNews++ )
			ReclasificarNoticia(*iterNews);		
		
		reclasPos[iter->getId()] += RECLASSIFY_STEP;
	}
}


void MainThread::Clean()
{
	state->giantLock->Enter();
	
        TInternalMap::iterator iteradorMapaCategorias;
        ConjuntoIdCategoria conjuntoCategoriasVivas;
        ListaConsultaFeed::iterator iteradorListaFeeds;
        ConjuntoFeeds conjuntoFeedVivos;

	LOG1("    Recovering tags map...");
        TInternalMap mapaCategoriasVivas = state->mapaCategorias->getMap();
	LOG1("    Recovering feed list...");
        ListaConsultaFeed listaFeedsVivos = state->feedManager->ConsultarFeed();

        for (	iteradorMapaCategorias = mapaCategoriasVivas.begin();
        	iteradorMapaCategorias != mapaCategoriasVivas.end();
		iteradorMapaCategorias++)
	{
	        conjuntoCategoriasVivas.insert (iteradorMapaCategorias->second);
	}

	for (	iteradorListaFeeds = listaFeedsVivos.begin();
	     	iteradorListaFeeds != listaFeedsVivos.end();
		iteradorListaFeeds++)
	{
	        conjuntoFeedVivos.insert(iteradorListaFeeds->getId());
	}

	LOG1("    Cleaning Tags Manager...");
	state->tagsManager->ReorganizarArchivos(conjuntoCategoriasVivas, *state->newsManager);
	LOG1("    Cleaning News Manager...");
	state->newsManager->ReorganizarArchivos(conjuntoCategoriasVivas, conjuntoFeedVivos);
	LOG1("    Cleaning Feed Manager...");
	state->feedManager->ReorganizarArchivos();

	state->giantLock->Release();
}


void cleanrequest_signal_handler(int sig_num)
{
	signal(SIGUSR1,cleanrequest_signal_handler);
	MainThread::SetCleanRequested();
}


void term_signal_handler(int sig_num)
{
	signal(SIGTERM,term_signal_handler);
	MainThread::SetPoliteStop();
}


void MainThread::main()
{
	sleep(1000);
	Parser parser(PLUGIN);

	signal(SIGUSR1,cleanrequest_signal_handler);
	signal(SIGTERM,term_signal_handler);
	
	while ( !MainThread::politeStop )
	{
		LOG1("Starting Main Thread iteration...");

		ParseFeeds(parser);
		Reclasificar();
		
		if ( MainThread::dirty )
		{
			LOG1("  Bayesian Filter dirty... Serializing...");
			
			state->giantLock->Enter();
			MainThread::dirty = false;
			BayesianFilterSerializer serializer(*state->bayesianFilter);
			serializer.Serialize(BAYES_DATA_FILE,BAYES_MATRIX_FILE);
			state->giantLock->Release();
		}

		if ( MainThread::cleanRequested )
		{
			LOG1("  Clean requested... starting...");
			Clean();
			MainThread::cleanRequested = false;
		}
		
		LOG1("Main Thread iteration finished.");
		if ( !MainThread::politeStop )
			sleep(MAINTHREAD_PERIOD);
	}

	// Para evitar cortar por la mitad una operacion en curso
	// pedida por el usuario a traves del xmlrpc
	state->giantLock->Enter();
	
	LOG1("Successful shutdown.");
	exit(0);
}
